package com.feidee.fd.sml.algorithm.feature

import com.feidee.fd.sml.algorithm.component.feature.{StopWordsRemoveEncoder, StopWordsRemoveEncoderParam}
import com.feidee.fd.sml.algorithm.util.{TestingDataGenerator, ToolClass}
import org.scalatest.FunSuite

/**
  * @Author JunxinWang
  * @Date 2019/3/25 20:16
  * @Description
  * @Reviewer YongChen
  */
class StopWordsRemoveEncoderSuite extends FunSuite {

  val stopWords = new StopWordsRemoveEncoder()
  val paramStr: String =
    """
      |{
      | "inputCol" : "feature",
      | "outputCol" : "abc" ,
      | "stopWords" : ["asd","abc"]
      |}
    """.stripMargin
  val param: StopWordsRemoveEncoderParam = stopWords.parseParam(new ToolClass().encrypt(paramStr))

  test("tokenizer parameter construction test") {
    assert(param.input_pt == null && param.output_pt == null && param.modelPath == null &&
      "feature".equals(param.inputCol)
    && !param.caseSensitive


    )

  }

  test("tokenizer transforming function test") {
    import TestingDataGenerator.spark.implicits._
    val rawData = Seq(Seq("abc","adad","asd"), Seq("asdasda","as", "das","d")).toDF("feature")

    val tokenizerData = stopWords.train(param,rawData).transform(rawData)
    tokenizerData.show()

  }

}
